


<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>torch.nn.parallel.distributed &mdash; PyTorch master documentation</title>
  

  
  
  
  
    <link rel="canonical" href="https://pytorch.org/docs/stable/_modules/torch/nn/parallel/distributed.html"/>
  

  

  
  
    

  

  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../../../_static/css/jit.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.11.1/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../../../_static/katex-math.css" type="text/css" />
    <link rel="index" title="Index" href="../../../../genindex.html" />
    <link rel="search" title="Search" href="../../../../search.html" /> 

  
  <script src="../../../../_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <div class="ecosystem-dropdown">
              <a id="dropdownMenuButton" data-toggle="ecosystem-dropdown">
                Ecosystem
              </a>
              <div class="ecosystem-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/hub"">
                  <span class=dropdown-title>Models (Beta)</span>
                  <p>Discover, publish, and reuse pre-trained models</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/ecosystem">
                  <span class=dropdown-title>Tools & Libraries</span>
                  <p>Explore the ecosystem of tools and libraries</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <div class="resources-dropdown">
              <a id="resourcesDropdownButton" data-toggle="resources-dropdown">
                Resources
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/resources"">
                  <span class=dropdown-title>Developer Resources</span>
                  <p>Find resources and get questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/features">
                  <span class=dropdown-title>About</span>
                  <p>Learn about PyTorch’s features and capabilities</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>

  </div>
</div>


<body class="pytorch-body">

   

    

    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            

            
              
              
                <div class="version">
                  master (1.5.0 )
                </div>
              
            

            


  


<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

            
          </div>

          
<div>
  <a style="color:#F05732" href="https://pytorch.org/docs/stable/_modules/torch/nn/parallel/distributed.html">
    You are viewing unstable developer preview docs.
    Click here to view docs for latest stable release.
  </a>
</div>

            
            
              
            
            
              <p class="caption"><span class="caption-text">Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/amp_examples.html">Automatic Mixed Precision examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/autograd.html">Autograd mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/broadcasting.html">Broadcasting semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/cpu_threading_torchscript_inference.html">CPU threading and TorchScript inference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/cuda.html">CUDA semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/ddp.html">Distributed Data Parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/extending.html">Extending PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/large_scale_deployments.html">Features for large-scale deployments</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/multiprocessing.html">Multiprocessing best practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/randomness.html">Reproducibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/serialization.html">Serialization semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/windows.html">Windows FAQ</a></li>
</ul>
<p class="caption"><span class="caption-text">Language Bindings</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/cppdocs/">C++ API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../packages.html">Javadoc</a></li>
</ul>
<p class="caption"><span class="caption-text">Python API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../torch.html">torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../nn.html">torch.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../nn.functional.html">torch.nn.functional</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensors.html">torch.Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensor_attributes.html">Tensor Attributes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensor_view.html">Tensor Views</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../autograd.html">torch.autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../cuda.html">torch.cuda</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../amp.html">torch.cuda.amp</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../distributed.html">torch.distributed</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../distributions.html">torch.distributions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../hub.html">torch.hub</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../jit.html">torch.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../nn.init.html">torch.nn.init</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../onnx.html">torch.onnx</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../optim.html">torch.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../quantization.html">Quantization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../rpc/index.html">Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../random.html">torch.random</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../sparse.html">torch.sparse</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../storage.html">torch.Storage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../bottleneck.html">torch.utils.bottleneck</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../checkpoint.html">torch.utils.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../cpp_extension.html">torch.utils.cpp_extension</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../data.html">torch.utils.data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../dlpack.html">torch.utils.dlpack</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../model_zoo.html">torch.utils.model_zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensorboard.html">torch.utils.tensorboard</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../type_info.html">Type Info</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../named_tensor.html">Named Tensors</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../name_inference.html">Named Tensors operator coverage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../__config__.html">torch.__config__</a></li>
</ul>
<p class="caption"><span class="caption-text">Libraries</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/audio">torchaudio</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/text">torchtext</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/elastic/">TorchElastic</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/serve">TorchServe</a></li>
<li class="toctree-l1"><a class="reference external" href="http://pytorch.org/xla/">PyTorch on XLA Devices</a></li>
</ul>
<p class="caption"><span class="caption-text">Community</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../community/contribution_guide.html">PyTorch Contribution Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../community/governance.html">PyTorch Governance</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../community/persons_of_interest.html">PyTorch Governance | Persons of Interest</a></li>
</ul>

            
          

        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="../../../../index.html">
          
            Docs
          
        </a> &gt;
      </li>

        
          <li><a href="../../../index.html">Module code</a> &gt;</li>
        
          <li><a href="../../../torch.html">torch</a> &gt;</li>
        
      <li>torch.nn.parallel.distributed</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <h1>Source code for torch.nn.parallel.distributed</h1><div class="highlight"><pre>
<span></span><span class="kn">from</span> <span class="nn">contextlib</span> <span class="kn">import</span> <span class="n">contextmanager</span>
<span class="kn">import</span> <span class="nn">copy</span>
<span class="kn">import</span> <span class="nn">itertools</span>

<span class="kn">import</span> <span class="nn">torch</span>

<span class="kn">import</span> <span class="nn">torch.cuda.comm</span>
<span class="kn">import</span> <span class="nn">torch.distributed</span> <span class="k">as</span> <span class="nn">dist</span>

<span class="k">if</span> <span class="n">dist</span><span class="o">.</span><span class="n">is_available</span><span class="p">():</span>
    <span class="kn">from</span> <span class="nn">torch.distributed.distributed_c10d</span> <span class="kn">import</span> <span class="n">_get_default_group</span>

<span class="kn">from</span> <span class="nn">..modules</span> <span class="kn">import</span> <span class="n">Module</span>
<span class="kn">from</span> <span class="nn">.replicate</span> <span class="kn">import</span> <span class="n">replicate</span>
<span class="kn">from</span> <span class="nn">.scatter_gather</span> <span class="kn">import</span> <span class="n">scatter_kwargs</span><span class="p">,</span> <span class="n">gather</span>
<span class="kn">from</span> <span class="nn">.parallel_apply</span> <span class="kn">import</span> <span class="n">parallel_apply</span>
<span class="kn">from</span> <span class="nn">torch.cuda._utils</span> <span class="kn">import</span> <span class="n">_get_device_index</span>


<span class="k">def</span> <span class="nf">_find_tensors</span><span class="p">(</span><span class="n">obj</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Recursively find all tensors contained in the specified object.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">Tensor</span><span class="p">):</span>
        <span class="k">return</span> <span class="p">[</span><span class="n">obj</span><span class="p">]</span>
    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
        <span class="k">return</span> <span class="n">itertools</span><span class="o">.</span><span class="n">chain</span><span class="p">(</span><span class="o">*</span><span class="nb">map</span><span class="p">(</span><span class="n">_find_tensors</span><span class="p">,</span> <span class="n">obj</span><span class="p">))</span>
    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">obj</span><span class="p">,</span> <span class="nb">dict</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">itertools</span><span class="o">.</span><span class="n">chain</span><span class="p">(</span><span class="o">*</span><span class="nb">map</span><span class="p">(</span><span class="n">_find_tensors</span><span class="p">,</span> <span class="n">obj</span><span class="o">.</span><span class="n">values</span><span class="p">()))</span>
    <span class="k">return</span> <span class="p">[]</span>


<div class="viewcode-block" id="DistributedDataParallel"><a class="viewcode-back" href="../../../../nn.html#torch.nn.parallel.DistributedDataParallel">[docs]</a><span class="k">class</span> <span class="nc">DistributedDataParallel</span><span class="p">(</span><span class="n">Module</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Implements distributed data parallelism that is based on</span>
<span class="sd">    ``torch.distributed`` package at the module level.</span>

<span class="sd">    This container parallelizes the application of the given module by</span>
<span class="sd">    splitting the input across the specified devices by chunking in the batch</span>
<span class="sd">    dimension. The module is replicated on each machine and each device, and</span>
<span class="sd">    each such replica handles a portion of the input. During the backwards</span>
<span class="sd">    pass, gradients from each node are averaged.</span>

<span class="sd">    The batch size should be larger than the number of GPUs used locally.</span>

<span class="sd">    See also: :ref:`distributed-basics` and :ref:`cuda-nn-ddp-instead`.</span>
<span class="sd">    The same constraints on input as in :class:`torch.nn.DataParallel` apply.</span>

<span class="sd">    Creation of this class requires that ``torch.distributed`` to be already</span>
<span class="sd">    initialized, by calling :func:`torch.distributed.init_process_group`.</span>

<span class="sd">    ``DistributedDataParallel`` is proven to be significantly faster than</span>
<span class="sd">    :class:`torch.nn.DataParallel` for single-node multi-GPU data</span>
<span class="sd">    parallel training.</span>

<span class="sd">    Here is how to use it: on each host with N GPUs, you should spawn up N</span>
<span class="sd">    processes, while ensuring that each process individually works on a single GPU</span>
<span class="sd">    from 0 to N-1. Therefore, it is your job to ensure that your training script</span>
<span class="sd">    operates on a single given GPU by calling:</span>

<span class="sd">        &gt;&gt;&gt; torch.cuda.set_device(i)</span>

<span class="sd">    where i is from 0 to N-1. In each process, you should refer the following</span>
<span class="sd">    to construct this module:</span>

<span class="sd">        &gt;&gt;&gt; torch.distributed.init_process_group(backend=&#39;nccl&#39;, world_size=4, init_method=&#39;...&#39;)</span>
<span class="sd">        &gt;&gt;&gt; model = DistributedDataParallel(model, device_ids=[i], output_device=i)</span>

<span class="sd">    In order to spawn up multiple processes per node, you can use either</span>
<span class="sd">    ``torch.distributed.launch`` or ``torch.multiprocessing.spawn``</span>

<span class="sd">    .. note:: ``nccl`` backend is currently the fastest and</span>
<span class="sd">        highly recommended backend to be used with Multi-Process Single-GPU</span>
<span class="sd">        distributed training and this applies to both single-node and multi-node</span>
<span class="sd">        distributed training</span>

<span class="sd">    .. note:: This module also supports mixed-precision distributed training.</span>
<span class="sd">        This means that your model can have different types of parameters such</span>
<span class="sd">        as mixed types of fp16 and fp32, the gradient reduction on these</span>
<span class="sd">        mixed types of parameters will just work fine.</span>
<span class="sd">        Also note that ``nccl`` backend is currently the fastest and highly</span>
<span class="sd">        recommended backend for fp16/fp32 mixed-precision training.</span>

<span class="sd">    .. note:: If you use ``torch.save`` on one process to checkpoint the module,</span>
<span class="sd">        and ``torch.load`` on some other processes to recover it, make sure that</span>
<span class="sd">        ``map_location`` is configured properly for every process. Without</span>
<span class="sd">        ``map_location``, ``torch.load`` would recover the module to devices</span>
<span class="sd">        where the module was saved from.</span>

<span class="sd">    .. warning::</span>
<span class="sd">        This module works only with the ``gloo`` and ``nccl`` backends.</span>

<span class="sd">    .. warning::</span>
<span class="sd">        Constructor, forward method, and differentiation of the output (or a</span>
<span class="sd">        function of the output of this module) is a distributed synchronization</span>
<span class="sd">        point. Take that into account in case different processes might be</span>
<span class="sd">        executing different code.</span>

<span class="sd">    .. warning::</span>
<span class="sd">        This module assumes all parameters are registered in the model by the</span>
<span class="sd">        time it is created. No parameters should be added nor removed later.</span>
<span class="sd">        Same applies to buffers.</span>

<span class="sd">    .. warning::</span>
<span class="sd">        This module assumes all parameters are registered in the model of each</span>
<span class="sd">        distributed processes are in the same order. The module itself will</span>
<span class="sd">        conduct gradient all-reduction following the reverse order of the</span>
<span class="sd">        registered parameters of the model. In other words, it is users&#39;</span>
<span class="sd">        responsibility to ensure that each distributed process has the exact</span>
<span class="sd">        same model and thus the exact same parameter registration order.</span>

<span class="sd">    .. warning::</span>
<span class="sd">        This module assumes all buffers and gradients are dense.</span>

<span class="sd">    .. warning::</span>
<span class="sd">        This module doesn&#39;t work with :func:`torch.autograd.grad` (i.e. it will</span>
<span class="sd">        only work if gradients are to be accumulated in ``.grad`` attributes of</span>
<span class="sd">        parameters).</span>

<span class="sd">    .. warning::</span>

<span class="sd">        If you plan on using this module with a ``nccl`` backend or a ``gloo``</span>
<span class="sd">        backend (that uses Infiniband), together with a DataLoader that uses</span>
<span class="sd">        multiple workers, please change the multiprocessing start method to</span>
<span class="sd">        ``forkserver`` (Python 3 only) or ``spawn``. Unfortunately</span>
<span class="sd">        Gloo (that uses Infiniband) and NCCL2 are not fork safe, and you will</span>
<span class="sd">        likely experience deadlocks if you don&#39;t change this setting.</span>

<span class="sd">    .. warning::</span>
<span class="sd">        Forward and backward hooks defined on :attr:`module` and its submodules</span>
<span class="sd">        won&#39;t be invoked anymore, unless the hooks are initialized in the</span>
<span class="sd">        :meth:`forward` method.</span>

<span class="sd">    .. warning::</span>
<span class="sd">        You should never try to change your model&#39;s parameters after wrapping</span>
<span class="sd">        up your model with DistributedDataParallel. In other words, when</span>
<span class="sd">        wrapping up your model with DistributedDataParallel, the constructor of</span>
<span class="sd">        DistributedDataParallel will register the additional gradient</span>
<span class="sd">        reduction functions on all the parameters of the model itself at the</span>
<span class="sd">        time of construction. If you change the model&#39;s parameters after</span>
<span class="sd">        the DistributedDataParallel construction, this is not supported and</span>
<span class="sd">        unexpected behaviors can happen, since some parameters&#39; gradient</span>
<span class="sd">        reduction functions might not get called.</span>

<span class="sd">    .. note::</span>
<span class="sd">        Parameters are never broadcast between processes. The module performs</span>
<span class="sd">        an all-reduce step on gradients and assumes that they will be modified</span>
<span class="sd">        by the optimizer in all processes in the same way. Buffers</span>
<span class="sd">        (e.g. BatchNorm stats) are broadcast from the module in process of rank</span>
<span class="sd">        0, to all other replicas in the system in every iteration.</span>

<span class="sd">    Args:</span>
<span class="sd">        module (Module): module to be parallelized</span>
<span class="sd">        device_ids (list of int or torch.device): CUDA devices. This should</span>
<span class="sd">                   only be provided when the input module resides on a single</span>
<span class="sd">                   CUDA device. For single-device modules, the ``i``th</span>
<span class="sd">                   :attr:`module` replica is placed on ``device_ids[i]``. For</span>
<span class="sd">                   multi-device modules and CPU modules, device_ids must be None</span>
<span class="sd">                   or an empty list, and input data for the forward pass must be</span>
<span class="sd">                   placed on the correct device. (default: all devices for</span>
<span class="sd">                   single-device modules)</span>
<span class="sd">        output_device (int or torch.device): device location of output for</span>
<span class="sd">                      single-device CUDA modules. For multi-device modules and</span>
<span class="sd">                      CPU modules, it must be None, and the module itself</span>
<span class="sd">                      dictates the output location. (default: device_ids[0] for</span>
<span class="sd">                      single-device modules)</span>
<span class="sd">        broadcast_buffers (bool): flag that enables syncing (broadcasting) buffers of</span>
<span class="sd">                          the module at beginning of the forward function.</span>
<span class="sd">                          (default: ``True``)</span>
<span class="sd">        process_group: the process group to be used for distributed data</span>
<span class="sd">                       all-reduction. If ``None``, the default process group, which</span>
<span class="sd">                       is created by ```torch.distributed.init_process_group```,</span>
<span class="sd">                       will be used. (default: ``None``)</span>
<span class="sd">        bucket_cap_mb: DistributedDataParallel will bucket parameters into</span>
<span class="sd">                       multiple buckets so that gradient reduction of each</span>
<span class="sd">                       bucket can potentially overlap with backward computation.</span>
<span class="sd">                       :attr:`bucket_cap_mb` controls the bucket size in MegaBytes (MB)</span>
<span class="sd">                       (default: 25)</span>
<span class="sd">        find_unused_parameters (bool): Traverse the autograd graph of all tensors</span>
<span class="sd">                                       contained in the return value of the wrapped</span>
<span class="sd">                                       module&#39;s ``forward`` function.</span>
<span class="sd">                                       Parameters that don&#39;t receive gradients as</span>
<span class="sd">                                       part of this graph are preemptively marked</span>
<span class="sd">                                       as being ready to be reduced. Note that all</span>
<span class="sd">                                       ``forward`` outputs that are derived from</span>
<span class="sd">                                       module parameters must participate in</span>
<span class="sd">                                       calculating loss and later the gradient</span>
<span class="sd">                                       computation. If they don&#39;t, this wrapper will</span>
<span class="sd">                                       hang waiting for autograd to produce gradients</span>
<span class="sd">                                       for those parameters. Any outputs derived from</span>
<span class="sd">                                       module parameters that are otherwise unused can</span>
<span class="sd">                                       be detached from the autograd graph using</span>
<span class="sd">                                       ``torch.Tensor.detach``. (default: ``False``)</span>
<span class="sd">        check_reduction: when setting to ``True``, it enables DistributedDataParallel</span>
<span class="sd">                         to automatically check if the previous iteration&#39;s</span>
<span class="sd">                         backward reductions were successfully issued at the</span>
<span class="sd">                         beginning of every iteration&#39;s forward function.</span>
<span class="sd">                         You normally don&#39;t need this option enabled unless you</span>
<span class="sd">                         are observing weird behaviors such as different ranks</span>
<span class="sd">                         are getting different gradients, which should not</span>
<span class="sd">                         happen if DistributedDataParallel is correctly used.</span>
<span class="sd">                         (default: ``False``)</span>

<span class="sd">    Attributes:</span>
<span class="sd">        module (Module): the module to be parallelized</span>

<span class="sd">    Example::</span>

<span class="sd">        &gt;&gt;&gt; torch.distributed.init_process_group(backend=&#39;nccl&#39;, world_size=4, init_method=&#39;...&#39;)</span>
<span class="sd">        &gt;&gt;&gt; net = torch.nn.DistributedDataParallel(model, pg)</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">module</span><span class="p">,</span> <span class="n">device_ids</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">output_device</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">broadcast_buffers</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                 <span class="n">process_group</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">bucket_cap_mb</span><span class="o">=</span><span class="mi">25</span><span class="p">,</span>
                 <span class="n">find_unused_parameters</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
                 <span class="n">check_reduction</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>

        <span class="nb">super</span><span class="p">(</span><span class="n">DistributedDataParallel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>

        <span class="k">assert</span> <span class="nb">any</span><span class="p">((</span><span class="n">p</span><span class="o">.</span><span class="n">requires_grad</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">module</span><span class="o">.</span><span class="n">parameters</span><span class="p">())),</span> <span class="p">(</span>
            <span class="s2">&quot;DistributedDataParallel is not needed when a module &quot;</span>
            <span class="s2">&quot;doesn&#39;t have any parameter that requires a gradient.&quot;</span>
        <span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">is_multi_device_module</span> <span class="o">=</span> <span class="nb">len</span><span class="p">({</span><span class="n">p</span><span class="o">.</span><span class="n">device</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">module</span><span class="o">.</span><span class="n">parameters</span><span class="p">()})</span> <span class="o">&gt;</span> <span class="mi">1</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">is_cuda</span> <span class="o">=</span> <span class="nb">all</span><span class="p">([</span><span class="n">p</span><span class="o">.</span><span class="n">device</span><span class="o">.</span><span class="n">type</span> <span class="o">==</span> <span class="s1">&#39;cuda&#39;</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">module</span><span class="o">.</span><span class="n">parameters</span><span class="p">()])</span>

        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_cuda</span> <span class="ow">or</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_multi_device_module</span><span class="p">:</span>
            <span class="k">assert</span> <span class="ow">not</span> <span class="n">device_ids</span> <span class="ow">and</span> <span class="ow">not</span> <span class="n">output_device</span><span class="p">,</span> <span class="p">(</span>
                <span class="s2">&quot;DistributedDataParallel device_ids and output_device arguments &quot;</span>
                <span class="s2">&quot;only work with single-device CUDA modules, but got &quot;</span>
                <span class="s2">&quot;device_ids </span><span class="si">{}</span><span class="s2">, output_device </span><span class="si">{}</span><span class="s2">, and module parameters </span><span class="si">{}</span><span class="s2">.&quot;</span>
            <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">device_ids</span><span class="p">,</span> <span class="n">output_device</span><span class="p">,</span> <span class="p">{</span><span class="n">p</span><span class="o">.</span><span class="n">device</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">module</span><span class="o">.</span><span class="n">parameters</span><span class="p">()})</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span> <span class="o">=</span> <span class="kc">None</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">output_device</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="c1"># Use all devices by default for single-device CUDA modules</span>
            <span class="k">if</span> <span class="n">device_ids</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">device_ids</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">range</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">device_count</span><span class="p">()))</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">_get_device_index</span><span class="p">(</span><span class="n">x</span><span class="p">,</span> <span class="kc">True</span><span class="p">),</span> <span class="n">device_ids</span><span class="p">))</span>

            <span class="k">if</span> <span class="n">output_device</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
                <span class="n">output_device</span> <span class="o">=</span> <span class="n">device_ids</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span>

            <span class="bp">self</span><span class="o">.</span><span class="n">output_device</span> <span class="o">=</span> <span class="n">_get_device_index</span><span class="p">(</span><span class="n">output_device</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_multi_device_module</span><span class="p">:</span>
            <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_cuda</span><span class="p">,</span> <span class="p">(</span>
                <span class="s2">&quot;DistributedDataParallel with multi-device module only works &quot;</span>
                <span class="s2">&quot;with CUDA devices, but module parameters locate in </span><span class="si">{}</span><span class="s2">.&quot;</span>
            <span class="p">)</span><span class="o">.</span><span class="n">format</span><span class="p">({</span><span class="n">p</span><span class="o">.</span><span class="n">device</span> <span class="k">for</span> <span class="n">p</span> <span class="ow">in</span> <span class="n">module</span><span class="o">.</span><span class="n">parameters</span><span class="p">()})</span>

        <span class="k">if</span> <span class="n">process_group</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">process_group</span> <span class="o">=</span> <span class="n">_get_default_group</span><span class="p">()</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">process_group</span> <span class="o">=</span> <span class="n">process_group</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">dim</span> <span class="o">=</span> <span class="n">dim</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">module</span> <span class="o">=</span> <span class="n">module</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">broadcast_buffers</span> <span class="o">=</span> <span class="n">broadcast_buffers</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">find_unused_parameters</span> <span class="o">=</span> <span class="n">find_unused_parameters</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">require_backward_grad_sync</span> <span class="o">=</span> <span class="kc">True</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">require_forward_param_sync</span> <span class="o">=</span> <span class="kc">True</span>

        <span class="k">if</span> <span class="n">check_reduction</span><span class="p">:</span>
            <span class="c1"># This argument is no longer used since the reducer</span>
            <span class="c1"># will ensure reduction completes even if some parameters</span>
            <span class="c1"># do not receive gradients.</span>
            <span class="k">pass</span>

        <span class="n">MB</span> <span class="o">=</span> <span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span>

        <span class="c1"># used for intra-node param sync and inter-node sync as well</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">broadcast_bucket_size</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="mi">250</span> <span class="o">*</span> <span class="n">MB</span><span class="p">)</span>

        <span class="c1"># reduction bucket size</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">bucket_bytes_cap</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">bucket_cap_mb</span> <span class="o">*</span> <span class="n">MB</span><span class="p">)</span>

        <span class="c1"># Sync params and buffers</span>
        <span class="n">module_states</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="o">.</span><span class="n">state_dict</span><span class="p">()</span><span class="o">.</span><span class="n">values</span><span class="p">())</span>
        <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">module_states</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_distributed_broadcast_coalesced</span><span class="p">(</span>
                <span class="n">module_states</span><span class="p">,</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">broadcast_bucket_size</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_ddp_init_helper</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">_ddp_init_helper</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        Initialization helper function that does the following:</span>

<span class="sd">        (1) replicating the module from device[0] to the other devices</span>
<span class="sd">        (2) bucketing the parameters for reductions</span>
<span class="sd">        (3) resetting the bucketing states</span>
<span class="sd">        (4) registering the grad hooks</span>
<span class="sd">        (5) passing a handle of DDP to SyncBatchNorm Layer</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>

            <span class="kn">import</span> <span class="nn">warnings</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span>
                <span class="s2">&quot;Single-Process Multi-GPU is not the recommended mode for &quot;</span>
                <span class="s2">&quot;DDP. In this mode, each DDP instance operates on multiple &quot;</span>
                <span class="s2">&quot;devices and creates multiple module replicas within one &quot;</span>
                <span class="s2">&quot;process. The overhead of scatter/gather and GIL contention &quot;</span>
                <span class="s2">&quot;in every forward pass can slow down training. &quot;</span>
                <span class="s2">&quot;Please consider using one DDP instance per device or per &quot;</span>
                <span class="s2">&quot;module replica by explicitly setting device_ids or &quot;</span>
                <span class="s2">&quot;CUDA_VISIBLE_DEVICES. &quot;</span>
                <span class="s2">&quot;NB: There is a known issue in nn.parallel.replicate that &quot;</span>
                <span class="s2">&quot;prevents a single DDP instance to operate on multiple model &quot;</span>
                <span class="s2">&quot;replicas.&quot;</span>
            <span class="p">)</span>

            <span class="c1"># only create replicas for single-device CUDA modules</span>
            <span class="c1">#</span>
            <span class="c1"># TODO: we don&#39;t need to replicate params in here. they&#39;re always going to</span>
            <span class="c1"># be broadcasted using larger blocks in broadcast_coalesced, so it might be</span>
            <span class="c1"># better to not pollute the caches with these small blocks</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_module_copies</span> <span class="o">=</span> <span class="n">replicate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span><span class="p">,</span> <span class="n">detach</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_module_copies</span><span class="p">[</span><span class="mi">0</span><span class="p">]</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">module</span>

            <span class="k">for</span> <span class="n">module_copy</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_module_copies</span><span class="p">[</span><span class="mi">1</span><span class="p">:]:</span>
                <span class="k">for</span> <span class="n">param</span><span class="p">,</span> <span class="n">copy_param</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="o">.</span><span class="n">parameters</span><span class="p">(),</span> <span class="n">module_copy</span><span class="o">.</span><span class="n">parameters</span><span class="p">()):</span>
                    <span class="n">copy_param</span><span class="o">.</span><span class="n">requires_grad</span> <span class="o">=</span> <span class="n">param</span><span class="o">.</span><span class="n">requires_grad</span>

        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_module_copies</span> <span class="o">=</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">]</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">modules_params</span> <span class="o">=</span> <span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="n">m</span><span class="o">.</span><span class="n">parameters</span><span class="p">())</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_module_copies</span><span class="p">]</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">modules_buffers</span> <span class="o">=</span> <span class="p">[</span><span class="nb">list</span><span class="p">(</span><span class="n">m</span><span class="o">.</span><span class="n">buffers</span><span class="p">())</span> <span class="k">for</span> <span class="n">m</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_module_copies</span><span class="p">]</span>

        <span class="c1"># Build tuple of (module, parameter) for all parameters that require grads.</span>
        <span class="n">modules_and_parameters</span> <span class="o">=</span> <span class="p">[</span>
            <span class="p">[</span>
                <span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="n">parameter</span><span class="p">)</span>
                <span class="k">for</span> <span class="n">module</span> <span class="ow">in</span> <span class="n">replica</span><span class="o">.</span><span class="n">modules</span><span class="p">()</span>
                <span class="k">for</span> <span class="n">parameter</span> <span class="ow">in</span> <span class="nb">filter</span><span class="p">(</span>
                    <span class="k">lambda</span> <span class="n">parameter</span><span class="p">:</span> <span class="n">parameter</span><span class="o">.</span><span class="n">requires_grad</span><span class="p">,</span>
                    <span class="n">module</span><span class="o">.</span><span class="n">parameters</span><span class="p">(</span><span class="n">recurse</span><span class="o">=</span><span class="kc">False</span><span class="p">))</span>
            <span class="p">]</span> <span class="k">for</span> <span class="n">replica</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_module_copies</span><span class="p">]</span>

        <span class="c1"># Build list of parameters.</span>
        <span class="n">parameters</span> <span class="o">=</span> <span class="p">[</span>
            <span class="nb">list</span><span class="p">(</span><span class="n">parameter</span> <span class="k">for</span> <span class="n">_</span><span class="p">,</span> <span class="n">parameter</span> <span class="ow">in</span> <span class="n">replica</span><span class="p">)</span>
            <span class="k">for</span> <span class="n">replica</span> <span class="ow">in</span> <span class="n">modules_and_parameters</span><span class="p">]</span>

        <span class="c1"># Checks if a module will produce a sparse gradient.</span>
        <span class="k">def</span> <span class="nf">produces_sparse_gradient</span><span class="p">(</span><span class="n">module</span><span class="p">):</span>
            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">Embedding</span><span class="p">):</span>
                <span class="k">return</span> <span class="n">module</span><span class="o">.</span><span class="n">sparse</span>
            <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">module</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">EmbeddingBag</span><span class="p">):</span>
                <span class="k">return</span> <span class="n">module</span><span class="o">.</span><span class="n">sparse</span>
            <span class="k">return</span> <span class="kc">False</span>

        <span class="c1"># Build list of booleans indicating whether or not to expect sparse</span>
        <span class="c1"># gradients for the corresponding parameters.</span>
        <span class="n">expect_sparse_gradient</span> <span class="o">=</span> <span class="p">[</span>
            <span class="nb">list</span><span class="p">(</span><span class="n">produces_sparse_gradient</span><span class="p">(</span><span class="n">module</span><span class="p">)</span> <span class="k">for</span> <span class="n">module</span><span class="p">,</span> <span class="n">_</span> <span class="ow">in</span> <span class="n">replica</span><span class="p">)</span>
            <span class="k">for</span> <span class="n">replica</span> <span class="ow">in</span> <span class="n">modules_and_parameters</span><span class="p">]</span>

        <span class="c1"># The bucket size limit is specified in the constructor.</span>
        <span class="c1"># Additionally, we allow for a single small bucket for parameters</span>
        <span class="c1"># that are defined first, such that their gradients don&#39;t spill into</span>
        <span class="c1"># a much larger bucket, adding unnecessary latency after gradient</span>
        <span class="c1"># computation finishes. Experiments showed 1MB is a reasonable value.</span>
        <span class="n">bucket_indices</span> <span class="o">=</span> <span class="n">dist</span><span class="o">.</span><span class="n">_compute_bucket_assignment_by_size</span><span class="p">(</span>
            <span class="n">parameters</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
            <span class="p">[</span><span class="mi">1024</span> <span class="o">*</span> <span class="mi">1024</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">bucket_bytes_cap</span><span class="p">],</span>
            <span class="n">expect_sparse_gradient</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>

        <span class="c1"># Note: reverse list of buckets because we want to approximate the</span>
        <span class="c1"># order in which their gradients are produced, and assume they</span>
        <span class="c1"># are used in the forward pass in the order they are defined.</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">reducer</span> <span class="o">=</span> <span class="n">dist</span><span class="o">.</span><span class="n">Reducer</span><span class="p">(</span>
            <span class="n">parameters</span><span class="p">,</span>
            <span class="nb">list</span><span class="p">(</span><span class="nb">reversed</span><span class="p">(</span><span class="n">bucket_indices</span><span class="p">)),</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">process_group</span><span class="p">,</span>
            <span class="n">expect_sparse_gradient</span><span class="p">)</span>

        <span class="c1"># passing a handle to torch.nn.SyncBatchNorm layer</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_passing_sync_batchnorm_handle</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_module_copies</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">__getstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_check_default_group</span><span class="p">()</span>
        <span class="n">attrs</span> <span class="o">=</span> <span class="n">copy</span><span class="o">.</span><span class="n">copy</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="p">)</span>
        <span class="k">del</span> <span class="n">attrs</span><span class="p">[</span><span class="s1">&#39;process_group&#39;</span><span class="p">]</span>
        <span class="k">del</span> <span class="n">attrs</span><span class="p">[</span><span class="s1">&#39;reducer&#39;</span><span class="p">]</span>
        <span class="k">return</span> <span class="n">attrs</span>

    <span class="k">def</span> <span class="nf">__setstate__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state</span><span class="p">):</span>
        <span class="c1"># If serializable, then the process group should be the default one</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">process_group</span> <span class="o">=</span> <span class="n">_get_default_group</span><span class="p">()</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">DistributedDataParallel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">__setstate__</span><span class="p">(</span><span class="n">state</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="s1">&#39;require_forward_param_sync&#39;</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="s1">&#39;require_backward_grad_sync&#39;</span><span class="p">,</span> <span class="kc">True</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_ddp_init_helper</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">_check_default_group</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="n">pickle_not_supported</span> <span class="o">=</span> <span class="kc">False</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">process_group</span> <span class="o">!=</span> <span class="n">_get_default_group</span><span class="p">():</span>
                <span class="n">pickle_not_supported</span> <span class="o">=</span> <span class="kc">True</span>
        <span class="k">except</span> <span class="ne">RuntimeError</span><span class="p">:</span>
            <span class="n">pickle_not_supported</span> <span class="o">=</span> <span class="kc">True</span>

        <span class="k">if</span> <span class="n">pickle_not_supported</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;DDP Pickling/Unpickling are only supported &quot;</span>
                               <span class="s2">&quot;when using DDP with the default process &quot;</span>
                               <span class="s2">&quot;group. That is, when you have called &quot;</span>
                               <span class="s2">&quot;init_process_group and have not passed &quot;</span>
                               <span class="s2">&quot;process_group argument to DDP constructor&quot;</span><span class="p">)</span>

<div class="viewcode-block" id="DistributedDataParallel.no_sync"><a class="viewcode-back" href="../../../../nn.html#torch.nn.parallel.DistributedDataParallel.no_sync">[docs]</a>    <span class="nd">@contextmanager</span>
    <span class="k">def</span> <span class="nf">no_sync</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">        A context manager to disable gradient synchronizations across DDP</span>
<span class="sd">        processes. Within this context, gradients will be accumulated on module</span>
<span class="sd">        variables, which will later be synchronized in the first</span>
<span class="sd">        forward-backward pass exiting the context.</span>

<span class="sd">        Example::</span>

<span class="sd">            &gt;&gt;&gt; ddp = torch.nn.DistributedDataParallel(model, pg)</span>
<span class="sd">            &gt;&gt;&gt; with ddp.no_sync():</span>
<span class="sd">            ...   for input in inputs:</span>
<span class="sd">            ...     ddp(input).backward()  # no synchronization, accumulate grads</span>
<span class="sd">            ... ddp(another_input).backward()  # synchronize grads</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">old_require_backward_grad_sync</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">require_backward_grad_sync</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">require_backward_grad_sync</span> <span class="o">=</span> <span class="kc">False</span>
        <span class="k">try</span><span class="p">:</span>
            <span class="k">yield</span>
        <span class="k">finally</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">require_backward_grad_sync</span> <span class="o">=</span> <span class="n">old_require_backward_grad_sync</span></div>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="o">*</span><span class="n">inputs</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">require_forward_param_sync</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_sync_params</span><span class="p">()</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span><span class="p">:</span>
            <span class="n">inputs</span><span class="p">,</span> <span class="n">kwargs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">scatter</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span><span class="p">)</span>
            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span><span class="p">)</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
                <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">(</span><span class="o">*</span><span class="n">inputs</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">outputs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">parallel_apply</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_module_copies</span><span class="p">[:</span><span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">)],</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">)</span>
                <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">gather</span><span class="p">(</span><span class="n">outputs</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">output_device</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">output</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">module</span><span class="p">(</span><span class="o">*</span><span class="n">inputs</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">is_grad_enabled</span><span class="p">()</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">require_backward_grad_sync</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">require_forward_param_sync</span> <span class="o">=</span> <span class="kc">True</span>
            <span class="c1"># We&#39;ll return the output object verbatim since it is a freeform</span>
            <span class="c1"># object. We need to find any tensors in this object, though,</span>
            <span class="c1"># because we need to figure out which parameters were used during</span>
            <span class="c1"># this forward pass, to ensure we short circuit reduction for any</span>
            <span class="c1"># unused parameters. Only if `find_unused_parameters` is set.</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">find_unused_parameters</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">reducer</span><span class="o">.</span><span class="n">prepare_for_backward</span><span class="p">(</span><span class="nb">list</span><span class="p">(</span><span class="n">_find_tensors</span><span class="p">(</span><span class="n">output</span><span class="p">)))</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">reducer</span><span class="o">.</span><span class="n">prepare_for_backward</span><span class="p">([])</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">require_forward_param_sync</span> <span class="o">=</span> <span class="kc">False</span>

        <span class="k">return</span> <span class="n">output</span>

    <span class="k">def</span> <span class="nf">scatter</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">,</span> <span class="n">device_ids</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">scatter_kwargs</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">,</span> <span class="n">device_ids</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dim</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">parallel_apply</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">replicas</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">parallel_apply</span><span class="p">(</span><span class="n">replicas</span><span class="p">,</span> <span class="n">inputs</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span><span class="p">[:</span><span class="nb">len</span><span class="p">(</span><span class="n">replicas</span><span class="p">)])</span>

    <span class="k">def</span> <span class="nf">gather</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">outputs</span><span class="p">,</span> <span class="n">output_device</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">gather</span><span class="p">(</span><span class="n">outputs</span><span class="p">,</span> <span class="n">output_device</span><span class="p">,</span> <span class="n">dim</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">dim</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">train</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">DistributedDataParallel</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="n">train</span><span class="p">(</span><span class="n">mode</span><span class="p">)</span>
        <span class="k">for</span> <span class="n">module</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_module_copies</span><span class="p">[</span><span class="mi">1</span><span class="p">:]:</span>
            <span class="n">module</span><span class="o">.</span><span class="n">train</span><span class="p">(</span><span class="n">mode</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_distributed_broadcast_coalesced</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">tensors</span><span class="p">,</span> <span class="n">buffer_size</span><span class="p">):</span>
        <span class="n">dist</span><span class="o">.</span><span class="n">_broadcast_coalesced</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">process_group</span><span class="p">,</span> <span class="n">tensors</span><span class="p">,</span> <span class="n">buffer_size</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_sync_params</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
            <span class="c1"># only do intra-node parameters sync for replicated single-device</span>
            <span class="c1"># CUDA modules</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
                <span class="c1"># intra-node parameter sync</span>
                <span class="n">result</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">comm</span><span class="o">.</span><span class="n">broadcast_coalesced</span><span class="p">(</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">modules_params</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span><span class="p">,</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">broadcast_bucket_size</span><span class="p">)</span>
                <span class="k">for</span> <span class="n">tensors</span><span class="p">,</span> <span class="n">module_params</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="mi">1</span><span class="p">:],</span>
                                                  <span class="bp">self</span><span class="o">.</span><span class="n">modules_params</span><span class="p">[</span><span class="mi">1</span><span class="p">:]):</span>
                    <span class="k">for</span> <span class="n">tensor</span><span class="p">,</span> <span class="n">param</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">tensors</span><span class="p">,</span> <span class="n">module_params</span><span class="p">):</span>
                        <span class="n">param</span><span class="o">.</span><span class="n">set_</span><span class="p">(</span><span class="n">tensor</span><span class="p">)</span>
                        <span class="c1"># Assume we have just run the optimizer and zeroed the</span>
                        <span class="c1"># grads of the parameters on the root model. We need</span>
                        <span class="c1"># to zero the grads on all model replicas as well.</span>
                        <span class="c1"># This snippet is copied from torch.optim.Optimizer.</span>
                        <span class="k">if</span> <span class="n">param</span><span class="o">.</span><span class="n">grad</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                            <span class="n">param</span><span class="o">.</span><span class="n">grad</span><span class="o">.</span><span class="n">detach_</span><span class="p">()</span>
                            <span class="n">param</span><span class="o">.</span><span class="n">grad</span><span class="o">.</span><span class="n">zero_</span><span class="p">()</span>

            <span class="c1"># module buffer sync</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">broadcast_buffers</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">modules_buffers</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
                <span class="c1"># Synchronize buffers across processes.</span>
                <span class="c1"># The process with rank 0 is considered the authoritative copy.</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">_distributed_broadcast_coalesced</span><span class="p">(</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">modules_buffers</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">broadcast_bucket_size</span><span class="p">)</span>
                <span class="c1"># only do intra-node buffer sync for replicated single-device</span>
                <span class="c1"># CUDA modules</span>
                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span> <span class="ow">and</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span><span class="p">)</span> <span class="o">&gt;</span> <span class="mi">1</span><span class="p">:</span>
                    <span class="c1"># intra-node buffer sync</span>
                    <span class="n">result</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">comm</span><span class="o">.</span><span class="n">broadcast_coalesced</span><span class="p">(</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">modules_buffers</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span><span class="p">,</span>
                        <span class="bp">self</span><span class="o">.</span><span class="n">broadcast_bucket_size</span><span class="p">)</span>
                    <span class="k">for</span> <span class="n">tensors</span><span class="p">,</span> <span class="n">module_buffers</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">result</span><span class="p">[</span><span class="mi">1</span><span class="p">:],</span>
                                                       <span class="bp">self</span><span class="o">.</span><span class="n">modules_buffers</span><span class="p">[</span><span class="mi">1</span><span class="p">:]):</span>
                        <span class="k">for</span> <span class="n">tensor</span><span class="p">,</span> <span class="n">buffer</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">tensors</span><span class="p">,</span> <span class="n">module_buffers</span><span class="p">):</span>
                            <span class="n">buffer</span><span class="o">.</span><span class="n">set_</span><span class="p">(</span><span class="n">tensor</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_passing_sync_batchnorm_handle</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">module_copies</span><span class="p">):</span>
        <span class="k">for</span> <span class="n">dev_idx</span><span class="p">,</span> <span class="n">module</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">module_copies</span><span class="p">):</span>
            <span class="k">for</span> <span class="n">layer</span> <span class="ow">in</span> <span class="n">module</span><span class="o">.</span><span class="n">modules</span><span class="p">():</span>
                <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">layer</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">nn</span><span class="o">.</span><span class="n">modules</span><span class="o">.</span><span class="n">SyncBatchNorm</span><span class="p">):</span>
                    <span class="k">assert</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_cuda</span><span class="p">,</span> <span class="s2">&quot;SyncBatchNorm layers only work with CUDA modules&quot;</span>
                    <span class="n">layer</span><span class="o">.</span><span class="n">_specify_ddp_gpu_num</span><span class="p">(</span>
                        <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span><span class="p">)</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">device_ids</span> <span class="k">else</span> <span class="mi">1</span><span class="p">)</span></div>
</pre></div>

             </article>
             
            </div>
            <footer>
  

  

    <hr>

  

  <div role="contentinfo">
    <p>
        &copy; Copyright 2019, Torch Contributors.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              
            </div>
          </div>
        </div>
      </section>
    </div>

  


  

     
       <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
         <script src="../../../../_static/jquery.js"></script>
         <script src="../../../../_static/underscore.js"></script>
         <script src="../../../../_static/doctools.js"></script>
         <script src="../../../../_static/language_data.js"></script>
     

  

  <script type="text/javascript" src="../../../../_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="../../../../_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>
 
<script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-90545585-1', 'auto');
  ga('send', 'pageview');

</script>

<script async src="https://www.googletagmanager.com/gtag/js?id=UA-117752657-2"></script>

<script>
  window.dataLayer = window.dataLayer || [];

  function gtag(){dataLayer.push(arguments);}

  gtag('js', new Date());
  gtag('config', 'UA-117752657-2');
</script>

<img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&amp;guid=ON&amp;script=0"/>


  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/resources">Resources</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.org/assets/brand-guidelines/PyTorch-Brand-Guidelines.pdf" target="_blank">Brand Guidelines</a></li>
          </ul>
        </div>

        <div class="footer-links-col follow-us-col">
          <ul>
            <li class="list-title">Stay Connected</li>
            <li>
              <div id="mc_embed_signup">
                <form
                  action="https://twitter.us14.list-manage.com/subscribe/post?u=75419c71fe0a935e53dfa4a3f&id=91d0dccd39"
                  method="post"
                  id="mc-embedded-subscribe-form"
                  name="mc-embedded-subscribe-form"
                  class="email-subscribe-form validate"
                  target="_blank"
                  novalidate>
                  <div id="mc_embed_signup_scroll" class="email-subscribe-form-fields-wrapper">
                    <div class="mc-field-group">
                      <label for="mce-EMAIL" style="display:none;">Email Address</label>
                      <input type="email" value="" name="EMAIL" class="required email" id="mce-EMAIL" placeholder="Email Address">
                    </div>

                    <div id="mce-responses" class="clear">
                      <div class="response" id="mce-error-response" style="display:none"></div>
                      <div class="response" id="mce-success-response" style="display:none"></div>
                    </div>    <!-- real people should not fill this in and expect good things - do not remove this or risk form bot signups-->

                    <div style="position: absolute; left: -5000px;" aria-hidden="true"><input type="text" name="b_75419c71fe0a935e53dfa4a3f_91d0dccd39" tabindex="-1" value=""></div>

                    <div class="clear">
                      <input type="submit" value="" name="subscribe" id="mc-embedded-subscribe" class="button email-subscribe-button">
                    </div>
                  </div>
                </form>
              </div>

            </li>
          </ul>

          <div class="footer-social-icons">
            <a href="https://www.facebook.com/pytorch" target="_blank" class="facebook"></a>
            <a href="https://twitter.com/pytorch" target="_blank" class="twitter"></a>
            <a href="https://www.youtube.com/pytorch" target="_blank" class="youtube"></a>
          </div>
        </div>
      </div>
    </div>
  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="../../../../_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/features">Features</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/hub">PyTorch Hub</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <a href="https://pytorch.org/resources">Resources</a>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="../../../../_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Remove any empty p tags that Sphinx adds
      $("[data-tags='null']").remove();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>